# -*- coding: utf-8 -*-
import multiprocessing
import time
from spider import shopeeSpider
from config import get_names
from Logs.log import log1


def run():
    sg = shopeeSpider()
    names = get_names()
    if not names:
        log1.info("请检查SHOPEE_STORELIST表数据或字段名")
        return
    n = 16
    pool = multiprocessing.Pool(processes=n)
    num = len(names) // n
    for i in range(n):
        ns = names[num * i:num * (i + 1)]
        if i == n - 1:
            ns = names[num * i:]
        pool.apply_async(sg.run, (ns,))
    pool.close()
    pool.join()


if __name__ == '__main__':
    start = time.time()
    print("----------------------------------正在爬取----------------------------------")
    log1.info("\n ----------------------------------正在爬取----------------------------------")
    run()
    end = time.time()
    print("抓取完毕, 共耗时:{}s".format(end-start))
    log1.info("抓取完毕, 共耗时:{}s".format(end-start))
